{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/functions.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def identity_function(x):\n",
    "    return x\n",
    "\n",
    "\n",
    "def step_function(x):\n",
    "    return np.array(x > 0, dtype=int)\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "\n",
    "def sigmoid_grad(x):\n",
    "    return (1.0 - sigmoid(x)) * sigmoid(x)\n",
    "\n",
    "\n",
    "def relu(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "\n",
    "def relu_grad(x):\n",
    "    grad = np.zeros_like(x)\n",
    "    grad[x>=0] = 1\n",
    "    return grad\n",
    "\n",
    "\n",
    "def softmax(x):\n",
    "    x = x - np.max(x, axis=-1, keepdims=True)   # オーバーフロー対策\n",
    "    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)\n",
    "\n",
    "\n",
    "def sum_squared_error(y, t):\n",
    "    return 0.5 * np.sum((y-t)**2)\n",
    "\n",
    "\n",
    "def cross_entropy_error(y, t):\n",
    "    if y.ndim == 1:\n",
    "        t = t.reshape(1, t.size)\n",
    "        y = y.reshape(1, y.size)\n",
    "\n",
    "    # 教師データがone-hot-vectorの場合、正解ラベルのインデックスに変換\n",
    "    if t.size == y.size:\n",
    "        t = t.argmax(axis=1)\n",
    "\n",
    "    batch_size = y.shape[0]\n",
    "    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size\n",
    "\n",
    "\n",
    "def softmax_loss(X, t):\n",
    "    y = softmax(X)\n",
    "    return cross_entropy_error(y, t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/gradient.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def _numerical_gradient_1d(f, x):\n",
    "    h = 1e-4 # 0.0001\n",
    "    grad = np.zeros_like(x)\n",
    "\n",
    "    for idx in range(x.size):\n",
    "        tmp_val = x[idx]\n",
    "        x[idx] = float(tmp_val) + h\n",
    "        fxh1 = f(x) # f(x+h)\n",
    "\n",
    "        x[idx] = tmp_val - h\n",
    "        fxh2 = f(x) # f(x-h)\n",
    "        grad[idx] = (fxh1 - fxh2) / (2*h)\n",
    "\n",
    "        x[idx] = tmp_val # 値を元に戻す\n",
    "\n",
    "    return grad\n",
    "\n",
    "\n",
    "def numerical_gradient_2d(f, X):\n",
    "    if X.ndim == 1:\n",
    "        return _numerical_gradient_1d(f, X)\n",
    "    else:\n",
    "        grad = np.zeros_like(X)\n",
    "\n",
    "        for idx, x in enumerate(X):\n",
    "            grad[idx] = _numerical_gradient_1d(f, x)\n",
    "\n",
    "        return grad\n",
    "\n",
    "\n",
    "def numerical_gradient(f, x):\n",
    "    h = 1e-4 # 0.0001\n",
    "    grad = np.zeros_like(x)\n",
    "\n",
    "    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])\n",
    "    while not it.finished:\n",
    "        idx = it.multi_index\n",
    "        tmp_val = x[idx]\n",
    "        x[idx] = tmp_val + h\n",
    "        fxh1 = f(x) # f(x+h)\n",
    "\n",
    "        x[idx] = tmp_val - h\n",
    "        fxh2 = f(x) # f(x-h)\n",
    "        grad[idx] = (fxh1 - fxh2) / (2*h)\n",
    "\n",
    "        x[idx] = tmp_val # 値を元に戻す\n",
    "        it.iternext()\n",
    "\n",
    "    return grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/layers.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class Relu:\n",
    "    def __init__(self):\n",
    "        self.mask = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        self.mask = (x <= 0)\n",
    "        out = x.copy()\n",
    "        out[self.mask] = 0\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dout[self.mask] = 0\n",
    "        dx = dout\n",
    "\n",
    "        return dx\n",
    "\n",
    "\n",
    "class Sigmoid:\n",
    "    def __init__(self):\n",
    "        self.out = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = sigmoid(x)\n",
    "        self.out = out\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * (1.0 - self.out) * self.out\n",
    "\n",
    "        return dx\n",
    "\n",
    "\n",
    "class Affine:\n",
    "    def __init__(self, W, b):\n",
    "        self.W =W\n",
    "        self.b = b\n",
    "\n",
    "        self.x = None\n",
    "        self.original_x_shape = None\n",
    "        # 重み・バイアスパラメータの微分\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        # テンソル対応\n",
    "        self.original_x_shape = x.shape\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        self.x = x\n",
    "\n",
    "        out = np.dot(self.x, self.W) + self.b\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = np.dot(dout, self.W.T)\n",
    "        self.dW = np.dot(self.x.T, dout)\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "\n",
    "        dx = dx.reshape(*self.original_x_shape)  # 入力データの形状に戻す（テンソル対応）\n",
    "        return dx\n",
    "\n",
    "\n",
    "class SoftmaxWithLoss:\n",
    "    def __init__(self):\n",
    "        self.loss = None\n",
    "        self.y = None # softmaxの出力\n",
    "        self.t = None # 教師データ\n",
    "\n",
    "    def forward(self, x, t):\n",
    "        self.t = t\n",
    "        self.y = softmax(x)\n",
    "        self.loss = cross_entropy_error(self.y, self.t)\n",
    "\n",
    "        return self.loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        batch_size = self.t.shape[0]\n",
    "        if self.t.size == self.y.size: # 教師データがone-hot-vectorの場合\n",
    "            dx = (self.y - self.t) / batch_size\n",
    "        else:\n",
    "            dx = self.y.copy()\n",
    "            dx[np.arange(batch_size), self.t] -= 1\n",
    "            dx = dx / batch_size\n",
    "\n",
    "        return dx\n",
    "\n",
    "\n",
    "class Dropout:\n",
    "    \"\"\"\n",
    "    http://arxiv.org/abs/1207.0580\n",
    "    \"\"\"\n",
    "    def __init__(self, dropout_ratio=0.5):\n",
    "        self.dropout_ratio = dropout_ratio\n",
    "        self.mask = None\n",
    "\n",
    "    def forward(self, x, train_flg=True):\n",
    "        if train_flg:\n",
    "            self.mask = np.random.rand(*x.shape) > self.dropout_ratio\n",
    "            return x * self.mask\n",
    "        else:\n",
    "            return x * (1.0 - self.dropout_ratio)\n",
    "\n",
    "    def backward(self, dout):\n",
    "        return dout * self.mask\n",
    "\n",
    "\n",
    "class BatchNormalization:\n",
    "    \"\"\"\n",
    "    http://arxiv.org/abs/1502.03167\n",
    "    \"\"\"\n",
    "    def __init__(self, gamma, beta, momentum=0.9, running_mean=None, running_var=None):\n",
    "        self.gamma = gamma\n",
    "        self.beta = beta\n",
    "        self.momentum = momentum\n",
    "        self.input_shape = None # Conv層の場合は4次元、全結合層の場合は2次元\n",
    "\n",
    "        # テスト時に使用する平均と分散\n",
    "        self.running_mean = running_mean\n",
    "        self.running_var = running_var\n",
    "\n",
    "        # backward時に使用する中間データ\n",
    "        self.batch_size = None\n",
    "        self.xc = None\n",
    "        self.std = None\n",
    "        self.dgamma = None\n",
    "        self.dbeta = None\n",
    "\n",
    "    def forward(self, x, train_flg=True):\n",
    "        self.input_shape = x.shape\n",
    "        if x.ndim != 2:\n",
    "            N, C, H, W = x.shape\n",
    "            x = x.reshape(N, -1)\n",
    "\n",
    "        out = self.__forward(x, train_flg)\n",
    "\n",
    "        return out.reshape(*self.input_shape)\n",
    "\n",
    "    def __forward(self, x, train_flg):\n",
    "        if self.running_mean is None:\n",
    "            N, D = x.shape\n",
    "            self.running_mean = np.zeros(D)\n",
    "            self.running_var = np.zeros(D)\n",
    "\n",
    "        if train_flg:\n",
    "            mu = x.mean(axis=0)\n",
    "            xc = x - mu\n",
    "            var = np.mean(xc**2, axis=0)\n",
    "            std = np.sqrt(var + 10e-7)\n",
    "            xn = xc / std\n",
    "\n",
    "            self.batch_size = x.shape[0]\n",
    "            self.xc = xc\n",
    "            self.xn = xn\n",
    "            self.std = std\n",
    "            self.running_mean = self.momentum * self.running_mean + (1-self.momentum) * mu\n",
    "            self.running_var = self.momentum * self.running_var + (1-self.momentum) * var\n",
    "        else:\n",
    "            xc = x - self.running_mean\n",
    "            xn = xc / ((np.sqrt(self.running_var + 10e-7)))\n",
    "\n",
    "        out = self.gamma * xn + self.beta\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        if dout.ndim != 2:\n",
    "            N, C, H, W = dout.shape\n",
    "            dout = dout.reshape(N, -1)\n",
    "\n",
    "        dx = self.__backward(dout)\n",
    "\n",
    "        dx = dx.reshape(*self.input_shape)\n",
    "        return dx\n",
    "\n",
    "    def __backward(self, dout):\n",
    "        dbeta = dout.sum(axis=0)\n",
    "        dgamma = np.sum(self.xn * dout, axis=0)\n",
    "        dxn = self.gamma * dout\n",
    "        dxc = dxn / self.std\n",
    "        dstd = -np.sum((dxn * self.xc) / (self.std * self.std), axis=0)\n",
    "        dvar = 0.5 * dstd / self.std\n",
    "        dxc += (2.0 / self.batch_size) * self.xc * dvar\n",
    "        dmu = np.sum(dxc, axis=0)\n",
    "        dx = dxc - dmu / self.batch_size\n",
    "\n",
    "        self.dgamma = dgamma\n",
    "        self.dbeta = dbeta\n",
    "\n",
    "        return dx\n",
    "\n",
    "\n",
    "class Convolution:\n",
    "    def __init__(self, W, b, stride=1, pad=0):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "\n",
    "        # 中間データ（backward時に使用）\n",
    "        self.x = None\n",
    "        self.col = None\n",
    "        self.col_W = None\n",
    "\n",
    "        # 重み・バイアスパラメータの勾配\n",
    "        self.dW = None\n",
    "        self.db = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        FN, C, FH, FW = self.W.shape\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = 1 + int((H + 2*self.pad - FH) / self.stride)\n",
    "        out_w = 1 + int((W + 2*self.pad - FW) / self.stride)\n",
    "\n",
    "        col = im2col(x, FH, FW, self.stride, self.pad)\n",
    "        col_W = self.W.reshape(FN, -1).T\n",
    "\n",
    "        out = np.dot(col, col_W) + self.b\n",
    "        out = out.reshape(N, out_h, out_w, -1).transpose(0, 3, 1, 2)\n",
    "\n",
    "        self.x = x\n",
    "        self.col = col\n",
    "        self.col_W = col_W\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        FN, C, FH, FW = self.W.shape\n",
    "        dout = dout.transpose(0,2,3,1).reshape(-1, FN)\n",
    "\n",
    "        self.db = np.sum(dout, axis=0)\n",
    "        self.dW = np.dot(self.col.T, dout)\n",
    "        self.dW = self.dW.transpose(1, 0).reshape(FN, C, FH, FW)\n",
    "\n",
    "        dcol = np.dot(dout, self.col_W.T)\n",
    "        dx = col2im(dcol, self.x.shape, FH, FW, self.stride, self.pad)\n",
    "\n",
    "        return dx\n",
    "\n",
    "\n",
    "class Pooling:\n",
    "    def __init__(self, pool_h, pool_w, stride=2, pad=0):\n",
    "        self.pool_h = pool_h\n",
    "        self.pool_w = pool_w\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "\n",
    "        self.x = None\n",
    "        self.arg_max = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = int(1 + (H - self.pool_h) / self.stride)\n",
    "        out_w = int(1 + (W - self.pool_w) / self.stride)\n",
    "\n",
    "        col = im2col(x, self.pool_h, self.pool_w, self.stride, self.pad)\n",
    "        col = col.reshape(-1, self.pool_h*self.pool_w)\n",
    "\n",
    "        arg_max = np.argmax(col, axis=1)\n",
    "        out = np.max(col, axis=1)\n",
    "        out = out.reshape(N, out_h, out_w, C).transpose(0, 3, 1, 2)\n",
    "\n",
    "        self.x = x\n",
    "        self.arg_max = arg_max\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dout = dout.transpose(0, 2, 3, 1)\n",
    "\n",
    "        pool_size = self.pool_h * self.pool_w\n",
    "        dmax = np.zeros((dout.size, pool_size))\n",
    "        dmax[np.arange(self.arg_max.size), self.arg_max.flatten()] = dout.flatten()\n",
    "        dmax = dmax.reshape(dout.shape + (pool_size,))\n",
    "\n",
    "        dcol = dmax.reshape(dmax.shape[0] * dmax.shape[1] * dmax.shape[2], -1)\n",
    "        dx = col2im(dcol, self.x.shape, self.pool_h, self.pool_w, self.stride, self.pad)\n",
    "\n",
    "        return dx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# common/optimizer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class SGD:\n",
    "\n",
    "    \"\"\"確率的勾配降下法（Stochastic Gradient Descent）\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        for key in params.keys():\n",
    "            params[key] -= self.lr * grads[key]\n",
    "\n",
    "\n",
    "class Momentum:\n",
    "\n",
    "    \"\"\"Momentum SGD\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.01, momentum=0.9):\n",
    "        self.lr = lr\n",
    "        self.momentum = momentum\n",
    "        self.v = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.v is None:\n",
    "            self.v = {}\n",
    "            for key, val in params.items():\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "\n",
    "        for key in params.keys():\n",
    "            self.v[key] = self.momentum*self.v[key] - self.lr*grads[key]\n",
    "            params[key] += self.v[key]\n",
    "\n",
    "\n",
    "class Nesterov:\n",
    "\n",
    "    \"\"\"Nesterov's Accelerated Gradient (http://arxiv.org/abs/1212.0901)\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.01, momentum=0.9):\n",
    "        self.lr = lr\n",
    "        self.momentum = momentum\n",
    "        self.v = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.v is None:\n",
    "            self.v = {}\n",
    "            for key, val in params.items():\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "\n",
    "        for key in params.keys():\n",
    "            params[key] += self.momentum * self.momentum * self.v[key]\n",
    "            params[key] -= (1 + self.momentum) * self.lr * grads[key]\n",
    "            self.v[key] *= self.momentum\n",
    "            self.v[key] -= self.lr * grads[key]\n",
    "\n",
    "\n",
    "class AdaGrad:\n",
    "\n",
    "    \"\"\"AdaGrad\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "        self.h = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.h is None:\n",
    "            self.h = {}\n",
    "            for key, val in params.items():\n",
    "                self.h[key] = np.zeros_like(val)\n",
    "\n",
    "        for key in params.keys():\n",
    "            self.h[key] += grads[key] * grads[key]\n",
    "            params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)\n",
    "\n",
    "\n",
    "class RMSprop:\n",
    "\n",
    "    \"\"\"RMSprop\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.01, decay_rate = 0.99):\n",
    "        self.lr = lr\n",
    "        self.decay_rate = decay_rate\n",
    "        self.h = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.h is None:\n",
    "            self.h = {}\n",
    "            for key, val in params.items():\n",
    "                self.h[key] = np.zeros_like(val)\n",
    "\n",
    "        for key in params.keys():\n",
    "            self.h[key] *= self.decay_rate\n",
    "            self.h[key] += (1 - self.decay_rate) * grads[key] * grads[key]\n",
    "            params[key] -= self.lr * grads[key] / (np.sqrt(self.h[key]) + 1e-7)\n",
    "\n",
    "\n",
    "class Adam:\n",
    "\n",
    "    \"\"\"Adam (http://arxiv.org/abs/1412.6980v8)\"\"\"\n",
    "\n",
    "    def __init__(self, lr=0.001, beta1=0.9, beta2=0.999):\n",
    "        self.lr = lr\n",
    "        self.beta1 = beta1\n",
    "        self.beta2 = beta2\n",
    "        self.iter = 0\n",
    "        self.m = None\n",
    "        self.v = None\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        if self.m is None:\n",
    "            self.m, self.v = {}, {}\n",
    "            for key, val in params.items():\n",
    "                self.m[key] = np.zeros_like(val)\n",
    "                self.v[key] = np.zeros_like(val)\n",
    "\n",
    "        self.iter += 1\n",
    "        lr_t  = self.lr * np.sqrt(1.0 - self.beta2**self.iter) / (1.0 - self.beta1**self.iter)\n",
    "\n",
    "        for key in params.keys():\n",
    "            #self.m[key] = self.beta1*self.m[key] + (1-self.beta1)*grads[key]\n",
    "            #self.v[key] = self.beta2*self.v[key] + (1-self.beta2)*(grads[key]**2)\n",
    "            self.m[key] += (1 - self.beta1) * (grads[key] - self.m[key])\n",
    "            self.v[key] += (1 - self.beta2) * (grads[key]**2 - self.v[key])\n",
    "\n",
    "            params[key] -= lr_t * self.m[key] / (np.sqrt(self.v[key]) + 1e-7)\n",
    "\n",
    "            #unbias_m += (1 - self.beta1) * (grads[key] - self.m[key]) # correct bias\n",
    "            #unbisa_b += (1 - self.beta2) * (grads[key]*grads[key] - self.v[key]) # correct bias\n",
    "            #params[key] += self.lr * unbias_m / (np.sqrt(unbisa_b) + 1e-7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/multi_layer_net.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir)  # 親ディレクトリのファイルをインポートするための設定\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "\n",
    "class MultiLayerNet:\n",
    "    \"\"\"全結合による多層ニューラルネットワーク\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    input_size : 入力サイズ（MNISTの場合は784）\n",
    "    hidden_size_list : 隠れ層のニューロンの数のリスト（e.g. [100, 100, 100]）\n",
    "    output_size : 出力サイズ（MNISTの場合は10）\n",
    "    activation : 'relu' or 'sigmoid'\n",
    "    weight_init_std : 重みの標準偏差を指定（e.g. 0.01）\n",
    "        'relu'または'he'を指定した場合は「Heの初期値」を設定\n",
    "        'sigmoid'または'xavier'を指定した場合は「Xavierの初期値」を設定\n",
    "    weight_decay_lambda : Weight Decay（L2ノルム）の強さ\n",
    "    \"\"\"\n",
    "    def __init__(self, input_size, hidden_size_list, output_size,\n",
    "                 activation='relu', weight_init_std='relu', weight_decay_lambda=0):\n",
    "        self.input_size = input_size\n",
    "        self.output_size = output_size\n",
    "        self.hidden_size_list = hidden_size_list\n",
    "        self.hidden_layer_num = len(hidden_size_list)\n",
    "        self.weight_decay_lambda = weight_decay_lambda\n",
    "        self.params = {}\n",
    "\n",
    "        # 重みの初期化\n",
    "        self.__init_weight(weight_init_std)\n",
    "\n",
    "        # レイヤの生成\n",
    "        activation_layer = {'sigmoid': Sigmoid, 'relu': Relu}\n",
    "        self.layers = OrderedDict()\n",
    "        for idx in range(1, self.hidden_layer_num+1):\n",
    "            self.layers['Affine' + str(idx)] = Affine(self.params['W' + str(idx)],\n",
    "                                                      self.params['b' + str(idx)])\n",
    "            self.layers['Activation_function' + str(idx)] = activation_layer[activation]()\n",
    "\n",
    "        idx = self.hidden_layer_num + 1\n",
    "        self.layers['Affine' + str(idx)] = Affine(self.params['W' + str(idx)],\n",
    "            self.params['b' + str(idx)])\n",
    "\n",
    "        self.last_layer = SoftmaxWithLoss()\n",
    "\n",
    "    def __init_weight(self, weight_init_std):\n",
    "        \"\"\"重みの初期値設定\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        weight_init_std : 重みの標準偏差を指定（e.g. 0.01）\n",
    "            'relu'または'he'を指定した場合は「Heの初期値」を設定\n",
    "            'sigmoid'または'xavier'を指定した場合は「Xavierの初期値」を設定\n",
    "        \"\"\"\n",
    "        all_size_list = [self.input_size] + self.hidden_size_list + [self.output_size]\n",
    "        for idx in range(1, len(all_size_list)):\n",
    "            scale = weight_init_std\n",
    "            if str(weight_init_std).lower() in ('relu', 'he'):\n",
    "                scale = np.sqrt(2.0 / all_size_list[idx - 1])  # ReLUを使う場合に推奨される初期値\n",
    "            elif str(weight_init_std).lower() in ('sigmoid', 'xavier'):\n",
    "                scale = np.sqrt(1.0 / all_size_list[idx - 1])  # sigmoidを使う場合に推奨される初期値\n",
    "\n",
    "            self.params['W' + str(idx)] = scale * np.random.randn(all_size_list[idx-1], all_size_list[idx])\n",
    "            self.params['b' + str(idx)] = np.zeros(all_size_list[idx])\n",
    "\n",
    "    def predict(self, x):\n",
    "        for layer in self.layers.values():\n",
    "            x = layer.forward(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def loss(self, x, t):\n",
    "        \"\"\"損失関数を求める\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : 入力データ\n",
    "        t : 教師ラベル\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        損失関数の値\n",
    "        \"\"\"\n",
    "        y = self.predict(x)\n",
    "\n",
    "        weight_decay = 0\n",
    "        for idx in range(1, self.hidden_layer_num + 2):\n",
    "            W = self.params['W' + str(idx)]\n",
    "            weight_decay += 0.5 * self.weight_decay_lambda * np.sum(W ** 2)\n",
    "\n",
    "        return self.last_layer.forward(y, t) + weight_decay\n",
    "\n",
    "    def accuracy(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        y = np.argmax(y, axis=1)\n",
    "        if t.ndim != 1 : t = np.argmax(t, axis=1)\n",
    "\n",
    "        accuracy = np.sum(y == t) / float(x.shape[0])\n",
    "        return accuracy\n",
    "\n",
    "    def numerical_gradient(self, x, t):\n",
    "        \"\"\"勾配を求める（数値微分）\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : 入力データ\n",
    "        t : 教師ラベル\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        各層の勾配を持ったディクショナリ変数\n",
    "            grads['W1']、grads['W2']、...は各層の重み\n",
    "            grads['b1']、grads['b2']、...は各層のバイアス\n",
    "        \"\"\"\n",
    "        loss_W = lambda W: self.loss(x, t)\n",
    "\n",
    "        grads = {}\n",
    "        for idx in range(1, self.hidden_layer_num+2):\n",
    "            grads['W' + str(idx)] = numerical_gradient(loss_W, self.params['W' + str(idx)])\n",
    "            grads['b' + str(idx)] = numerical_gradient(loss_W, self.params['b' + str(idx)])\n",
    "\n",
    "        return grads\n",
    "\n",
    "    def gradient(self, x, t):\n",
    "        \"\"\"勾配を求める（誤差逆伝搬法）\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : 入力データ\n",
    "        t : 教師ラベル\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        各層の勾配を持ったディクショナリ変数\n",
    "            grads['W1']、grads['W2']、...は各層の重み\n",
    "            grads['b1']、grads['b2']、...は各層のバイアス\n",
    "        \"\"\"\n",
    "        # forward\n",
    "        self.loss(x, t)\n",
    "\n",
    "        # backward\n",
    "        dout = 1\n",
    "        dout = self.last_layer.backward(dout)\n",
    "\n",
    "        layers = list(self.layers.values())\n",
    "        layers.reverse()\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout)\n",
    "\n",
    "        # 設定\n",
    "        grads = {}\n",
    "        for idx in range(1, self.hidden_layer_num+2):\n",
    "            grads['W' + str(idx)] = self.layers['Affine' + str(idx)].dW + self.weight_decay_lambda * self.layers['Affine' + str(idx)].W\n",
    "            grads['b' + str(idx)] = self.layers['Affine' + str(idx)].db\n",
    "\n",
    "        return grads\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# common/multi_layer_net_extend.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir) # 親ディレクトリのファイルをインポートするための設定\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "class MultiLayerNetExtend:\n",
    "    \"\"\"拡張版の全結合による多層ニューラルネットワーク\n",
    "\n",
    "    Weiht Decay、Dropout、Batch Normalizationの機能を持つ\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    input_size : 入力サイズ（MNISTの場合は784）\n",
    "    hidden_size_list : 隠れ層のニューロンの数のリスト（e.g. [100, 100, 100]）\n",
    "    output_size : 出力サイズ（MNISTの場合は10）\n",
    "    activation : 'relu' or 'sigmoid'\n",
    "    weight_init_std : 重みの標準偏差を指定（e.g. 0.01）\n",
    "        'relu'または'he'を指定した場合は「Heの初期値」を設定\n",
    "        'sigmoid'または'xavier'を指定した場合は「Xavierの初期値」を設定\n",
    "    weight_decay_lambda : Weight Decay（L2ノルム）の強さ\n",
    "    use_dropout: Dropoutを使用するかどうか\n",
    "    dropout_ration : Dropoutの割り合い\n",
    "    use_batchNorm: Batch Normalizationを使用するかどうか\n",
    "    \"\"\"\n",
    "    def __init__(self, input_size, hidden_size_list, output_size,\n",
    "                 activation='relu', weight_init_std='relu', weight_decay_lambda=0,\n",
    "                 use_dropout = False, dropout_ration = 0.5, use_batchnorm=False):\n",
    "        self.input_size = input_size\n",
    "        self.output_size = output_size\n",
    "        self.hidden_size_list = hidden_size_list\n",
    "        self.hidden_layer_num = len(hidden_size_list)\n",
    "        self.use_dropout = use_dropout\n",
    "        self.weight_decay_lambda = weight_decay_lambda\n",
    "        self.use_batchnorm = use_batchnorm\n",
    "        self.params = {}\n",
    "\n",
    "        # 重みの初期化\n",
    "        self.__init_weight(weight_init_std)\n",
    "\n",
    "        # レイヤの生成\n",
    "        activation_layer = {'sigmoid': Sigmoid, 'relu': Relu}\n",
    "        self.layers = OrderedDict()\n",
    "        for idx in range(1, self.hidden_layer_num+1):\n",
    "            self.layers['Affine' + str(idx)] = Affine(self.params['W' + str(idx)],\n",
    "                                                      self.params['b' + str(idx)])\n",
    "            if self.use_batchnorm:\n",
    "                self.params['gamma' + str(idx)] = np.ones(hidden_size_list[idx-1])\n",
    "                self.params['beta' + str(idx)] = np.zeros(hidden_size_list[idx-1])\n",
    "                self.layers['BatchNorm' + str(idx)] = BatchNormalization(self.params['gamma' + str(idx)], self.params['beta' + str(idx)])\n",
    "\n",
    "            self.layers['Activation_function' + str(idx)] = activation_layer[activation]()\n",
    "\n",
    "            if self.use_dropout:\n",
    "                self.layers['Dropout' + str(idx)] = Dropout(dropout_ration)\n",
    "\n",
    "        idx = self.hidden_layer_num + 1\n",
    "        self.layers['Affine' + str(idx)] = Affine(self.params['W' + str(idx)], self.params['b' + str(idx)])\n",
    "\n",
    "        self.last_layer = SoftmaxWithLoss()\n",
    "\n",
    "    def __init_weight(self, weight_init_std):\n",
    "        \"\"\"重みの初期値設定\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        weight_init_std : 重みの標準偏差を指定（e.g. 0.01）\n",
    "            'relu'または'he'を指定した場合は「Heの初期値」を設定\n",
    "            'sigmoid'または'xavier'を指定した場合は「Xavierの初期値」を設定\n",
    "        \"\"\"\n",
    "        all_size_list = [self.input_size] + self.hidden_size_list + [self.output_size]\n",
    "        for idx in range(1, len(all_size_list)):\n",
    "            scale = weight_init_std\n",
    "            if str(weight_init_std).lower() in ('relu', 'he'):\n",
    "                scale = np.sqrt(2.0 / all_size_list[idx - 1])  # ReLUを使う場合に推奨される初期値\n",
    "            elif str(weight_init_std).lower() in ('sigmoid', 'xavier'):\n",
    "                scale = np.sqrt(1.0 / all_size_list[idx - 1])  # sigmoidを使う場合に推奨される初期値\n",
    "            self.params['W' + str(idx)] = scale * np.random.randn(all_size_list[idx-1], all_size_list[idx])\n",
    "            self.params['b' + str(idx)] = np.zeros(all_size_list[idx])\n",
    "\n",
    "    def predict(self, x, train_flg=False):\n",
    "        for key, layer in self.layers.items():\n",
    "            if \"Dropout\" in key or \"BatchNorm\" in key:\n",
    "                x = layer.forward(x, train_flg)\n",
    "            else:\n",
    "                x = layer.forward(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def loss(self, x, t, train_flg=False):\n",
    "        \"\"\"損失関数を求める\n",
    "        引数のxは入力データ、tは教師ラベル\n",
    "        \"\"\"\n",
    "        y = self.predict(x, train_flg)\n",
    "\n",
    "        weight_decay = 0\n",
    "        for idx in range(1, self.hidden_layer_num + 2):\n",
    "            W = self.params['W' + str(idx)]\n",
    "            weight_decay += 0.5 * self.weight_decay_lambda * np.sum(W**2)\n",
    "\n",
    "        return self.last_layer.forward(y, t) + weight_decay\n",
    "\n",
    "    def accuracy(self, x, t):\n",
    "        y = self.predict(x, train_flg=False)\n",
    "        y = np.argmax(y, axis=1)\n",
    "        if t.ndim != 1 : t = np.argmax(t, axis=1)\n",
    "\n",
    "        accuracy = np.sum(y == t) / float(x.shape[0])\n",
    "        return accuracy\n",
    "\n",
    "    def numerical_gradient(self, x, t):\n",
    "        \"\"\"勾配を求める（数値微分）\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : 入力データ\n",
    "        t : 教師ラベル\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        各層の勾配を持ったディクショナリ変数\n",
    "            grads['W1']、grads['W2']、...は各層の重み\n",
    "            grads['b1']、grads['b2']、...は各層のバイアス\n",
    "        \"\"\"\n",
    "        loss_W = lambda W: self.loss(x, t, train_flg=True)\n",
    "\n",
    "        grads = {}\n",
    "        for idx in range(1, self.hidden_layer_num+2):\n",
    "            grads['W' + str(idx)] = numerical_gradient(loss_W, self.params['W' + str(idx)])\n",
    "            grads['b' + str(idx)] = numerical_gradient(loss_W, self.params['b' + str(idx)])\n",
    "\n",
    "            if self.use_batchnorm and idx != self.hidden_layer_num+1:\n",
    "                grads['gamma' + str(idx)] = numerical_gradient(loss_W, self.params['gamma' + str(idx)])\n",
    "                grads['beta' + str(idx)] = numerical_gradient(loss_W, self.params['beta' + str(idx)])\n",
    "\n",
    "        return grads\n",
    "\n",
    "    def gradient(self, x, t):\n",
    "        # forward\n",
    "        self.loss(x, t, train_flg=True)\n",
    "\n",
    "        # backward\n",
    "        dout = 1\n",
    "        dout = self.last_layer.backward(dout)\n",
    "\n",
    "        layers = list(self.layers.values())\n",
    "        layers.reverse()\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout)\n",
    "\n",
    "        # 設定\n",
    "        grads = {}\n",
    "        for idx in range(1, self.hidden_layer_num+2):\n",
    "            grads['W' + str(idx)] = self.layers['Affine' + str(idx)].dW + self.weight_decay_lambda * self.params['W' + str(idx)]\n",
    "            grads['b' + str(idx)] = self.layers['Affine' + str(idx)].db\n",
    "\n",
    "            if self.use_batchnorm and idx != self.hidden_layer_num+1:\n",
    "                grads['gamma' + str(idx)] = self.layers['BatchNorm' + str(idx)].dgamma\n",
    "                grads['beta' + str(idx)] = self.layers['BatchNorm' + str(idx)].dbeta\n",
    "\n",
    "        return grads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/trainer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir)  # 親ディレクトリのファイルをインポートするための設定\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class Trainer:\n",
    "    \"\"\"ニューラルネットの訓練を行うクラス\n",
    "    \"\"\"\n",
    "    def __init__(self, network, x_train, t_train, x_test, t_test,\n",
    "                 epochs=20, mini_batch_size=100,\n",
    "                 optimizer='SGD', optimizer_param={'lr':0.01},\n",
    "                 evaluate_sample_num_per_epoch=None, verbose=True):\n",
    "        self.network = network\n",
    "        self.verbose = verbose\n",
    "        self.x_train = x_train\n",
    "        self.t_train = t_train\n",
    "        self.x_test = x_test\n",
    "        self.t_test = t_test\n",
    "        self.epochs = epochs\n",
    "        self.batch_size = mini_batch_size\n",
    "        self.evaluate_sample_num_per_epoch = evaluate_sample_num_per_epoch\n",
    "\n",
    "        # optimizer\n",
    "        optimizer_class_dict = {'sgd':SGD, 'momentum':Momentum, 'nesterov':Nesterov,\n",
    "                                'adagrad':AdaGrad, 'rmsprop':RMSprop, 'adam':Adam}\n",
    "        self.optimizer = optimizer_class_dict[optimizer.lower()](**optimizer_param)\n",
    "\n",
    "        self.train_size = x_train.shape[0]\n",
    "        self.iter_per_epoch = max(self.train_size / mini_batch_size, 1)\n",
    "        self.max_iter = int(epochs * self.iter_per_epoch)\n",
    "        self.current_iter = 0\n",
    "        self.current_epoch = 0\n",
    "\n",
    "        self.train_loss_list = []\n",
    "        self.train_acc_list = []\n",
    "        self.test_acc_list = []\n",
    "\n",
    "    def train_step(self):\n",
    "        batch_mask = np.random.choice(self.train_size, self.batch_size)\n",
    "        x_batch = self.x_train[batch_mask]\n",
    "        t_batch = self.t_train[batch_mask]\n",
    "\n",
    "        grads = self.network.gradient(x_batch, t_batch)\n",
    "        self.optimizer.update(self.network.params, grads)\n",
    "\n",
    "        loss = self.network.loss(x_batch, t_batch)\n",
    "        self.train_loss_list.append(loss)\n",
    "        if self.verbose: print(\"train loss:\" + str(loss))\n",
    "\n",
    "        if self.current_iter % self.iter_per_epoch == 0:\n",
    "            self.current_epoch += 1\n",
    "\n",
    "            x_train_sample, t_train_sample = self.x_train, self.t_train\n",
    "            x_test_sample, t_test_sample = self.x_test, self.t_test\n",
    "            if not self.evaluate_sample_num_per_epoch is None:\n",
    "                t = self.evaluate_sample_num_per_epoch\n",
    "                x_train_sample, t_train_sample = self.x_train[:t], self.t_train[:t]\n",
    "                x_test_sample, t_test_sample = self.x_test[:t], self.t_test[:t]\n",
    "\n",
    "            train_acc = self.network.accuracy(x_train_sample, t_train_sample)\n",
    "            test_acc = self.network.accuracy(x_test_sample, t_test_sample)\n",
    "            self.train_acc_list.append(train_acc)\n",
    "            self.test_acc_list.append(test_acc)\n",
    "\n",
    "            if self.verbose: print(\"=== epoch:\" + str(self.current_epoch) + \", train acc:\" + str(train_acc) + \", test acc:\" + str(test_acc) + \" ===\")\n",
    "        self.current_iter += 1\n",
    "\n",
    "    def train(self):\n",
    "        for i in range(self.max_iter):\n",
    "            self.train_step()\n",
    "\n",
    "        test_acc = self.network.accuracy(self.x_test, self.t_test)\n",
    "\n",
    "        if self.verbose:\n",
    "            print(\"=============== Final Test Accuracy ===============\")\n",
    "            print(\"test acc:\" + str(test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# common/util.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def smooth_curve(x):\n",
    "    \"\"\"損失関数のグラフを滑らかにするために用いる\n",
    "\n",
    "    参考：http://glowingpython.blogspot.jp/2012/02/convolution-with-numpy.html\n",
    "    \"\"\"\n",
    "    window_len = 11\n",
    "    s = np.r_[x[window_len-1:0:-1], x, x[-1:-window_len:-1]]\n",
    "    w = np.kaiser(window_len, 2)\n",
    "    y = np.convolve(w/w.sum(), s, mode='valid')\n",
    "    return y[5:len(y)-5]\n",
    "\n",
    "\n",
    "def shuffle_dataset(x, t):\n",
    "    \"\"\"データセットのシャッフルを行う\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x : 訓練データ\n",
    "    t : 教師データ\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    x, t : シャッフルを行った訓練データと教師データ\n",
    "    \"\"\"\n",
    "    permutation = np.random.permutation(x.shape[0])\n",
    "    x = x[permutation,:] if x.ndim == 2 else x[permutation,:,:,:]\n",
    "    t = t[permutation]\n",
    "\n",
    "    return x, t\n",
    "\n",
    "def conv_output_size(input_size, filter_size, stride=1, pad=0):\n",
    "    return (input_size + 2*pad - filter_size) / stride + 1\n",
    "\n",
    "\n",
    "def im2col(input_data, filter_h, filter_w, stride=1, pad=0):\n",
    "    \"\"\"\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    input_data : (データ数, チャンネル, 高さ, 幅)の4次元配列からなる入力データ\n",
    "    filter_h : フィルターの高さ\n",
    "    filter_w : フィルターの幅\n",
    "    stride : ストライド\n",
    "    pad : パディング\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    col : 2次元配列\n",
    "    \"\"\"\n",
    "    N, C, H, W = input_data.shape\n",
    "    out_h = (H + 2*pad - filter_h)//stride + 1\n",
    "    out_w = (W + 2*pad - filter_w)//stride + 1\n",
    "\n",
    "    img = np.pad(input_data, [(0,0), (0,0), (pad, pad), (pad, pad)], 'constant')\n",
    "    col = np.zeros((N, C, filter_h, filter_w, out_h, out_w))\n",
    "\n",
    "    for y in range(filter_h):\n",
    "        y_max = y + stride*out_h\n",
    "        for x in range(filter_w):\n",
    "            x_max = x + stride*out_w\n",
    "            col[:, :, y, x, :, :] = img[:, :, y:y_max:stride, x:x_max:stride]\n",
    "\n",
    "    col = col.transpose(0, 4, 5, 1, 2, 3).reshape(N*out_h*out_w, -1)\n",
    "    return col\n",
    "\n",
    "\n",
    "def col2im(col, input_shape, filter_h, filter_w, stride=1, pad=0):\n",
    "    \"\"\"\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    col :\n",
    "    input_shape : 入力データの形状（例：(10, 1, 28, 28)）\n",
    "    filter_h :\n",
    "    filter_w\n",
    "    stride\n",
    "    pad\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "\n",
    "    \"\"\"\n",
    "    N, C, H, W = input_shape\n",
    "    out_h = (H + 2*pad - filter_h)//stride + 1\n",
    "    out_w = (W + 2*pad - filter_w)//stride + 1\n",
    "    col = col.reshape(N, out_h, out_w, C, filter_h, filter_w).transpose(0, 3, 4, 5, 1, 2)\n",
    "\n",
    "    img = np.zeros((N, C, H + 2*pad + stride - 1, W + 2*pad + stride - 1))\n",
    "    for y in range(filter_h):\n",
    "        y_max = y + stride*out_h\n",
    "        for x in range(filter_w):\n",
    "            x_max = x + stride*out_w\n",
    "            img[:, :, y:y_max:stride, x:x_max:stride] += col[:, :, y, x, :, :]\n",
    "\n",
    "    return img[:, :, pad:H + pad, pad:W + pad]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dezero:Python",
   "language": "python",
   "name": "conda-env-dezero-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc-autonumbering": true,
  "toc-showcode": true,
  "toc-showmarkdowntxt": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
